package com.eascs.spider.huangye88.actor;

import java.util.HashMap;
import java.util.Map;

import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.eascs.spider.huangye88.manager.CrawlerGuangdongManager;
import com.eascs.spider.huangye88.util.NetUtil;

public class Huangye88Start implements Runnable{
	private static Logger logger = LoggerFactory.getLogger(Huangye88Start.class);
	public static CrawlerGuangdongManager manager = CrawlerGuangdongManager.getInstance();
	public static void main(String[] args) {
		BasicCookieStore cookieStore = new BasicCookieStore();
		CloseableHttpClient client = NetUtil.initHttpClient(cookieStore);
		String basicUrl = "http://www.huangye88.com/search.html?type=sale&kw=印刷深圳&";
		for(int i=1;i<200;i++){
			//http://www.huangye88.com/search.html?type=sale&kw=印刷 深圳&page=3/
			//http://www.huangye88.com/search.html?type=sale&kw=印刷 深圳&
			Map<String, String> header = new HashMap<>();
			header.put("Referer", "http://www.huangye88.com/search.html?type=sale&kw=印刷 深圳&");
			String url = i==1?basicUrl:basicUrl+"page="+i;
			HttpGet httpGet = NetUtil.initHttpGet(url, header, null);
			String html = NetUtil.extractPageRawText(NetUtil.execute(client, httpGet));
			try {
				Document document = Jsoup.parse(html);
				Elements elements = document.getElementsByClass("content").get(0).getElementsByTag("li");
				for(Element element : elements){
					String targetUrl = element.getElementsByTag("a").get(0).attr("href");
					if(targetUrl.contains("huangye88.com")){
						System.out.println(targetUrl);
						//manager.addUrl(targetUrl);
					}
						
				}
			} catch (Exception e) {
				// TODO: handle exception
			}
		}
	}
	
	@Override
	public void run(){
		BasicCookieStore cookieStore = new BasicCookieStore();
		CloseableHttpClient client = NetUtil.initHttpClient(cookieStore);
		String basicUrl = "http://www.huangye88.com/search.html?kw=印刷+深圳&type=company&";
		for(int i=1;i<300;i++){
			String url = i==1?basicUrl:basicUrl+"page="+i;
			logger.info("============================================================第{}页==================================================================" ,i);
			Map<String, String> header = new HashMap<>();
			String referer = i==1?basicUrl:basicUrl+"page="+(i-1);
			header.put("Referer", referer);
			HttpGet httpGet = NetUtil.initHttpGet(url, header, null);
			String html = NetUtil.extractPageRawText(NetUtil.execute(client, httpGet));
			try {
				Document document = Jsoup.parse(html);
				Elements elements = document.getElementsByClass("content").get(0).getElementsByTag("li");
				for(Element element : elements){
					try {
						String targetUrl = element.getElementsByTag("a").get(0).attr("href");
						if(targetUrl.contains("b2b.huangye88")){
							manager.addUrl(targetUrl+"company_contact.html");
							logger.info("第{}页获取的目标url：{}" ,i ,targetUrl);
						}
					} catch (Exception e) {
						// TODO: handle exception
					}
						
				}
			} catch (Exception e) {
				// TODO: handle exception
			}
		}
	}
}